In [3]:
import numpy as np
import pandas as pd
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from tqdm import tqdm
import os
from os.path import isfile, join
import random
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras import layers
from tensorflow.keras import metrics
In [5]:
DIR = "/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m"
INPUT_CHANNELS = 3
TARGET_CHANNELS = 1
SIZE = 256
BATCH_SIZE = 32
In [7]:
mri_images_with_tumer = []
mri_images_without_tumer = []
mask_images_with_tumer = []
mask_images_without_tumer = []

patients = os.listdir(DIR)
for patient in tqdm(patients):
    if isfile(join(DIR, patient)) == False:
        images = os.listdir(join(DIR, patient))
        mask_images = list(filter(lambda x: x.find('mask') != -1, images))
        mri_images = list(filter(lambda x: x.find('mask') == -1, images))
        
        for mask_image in mask_images:
            mask = np.asarray(load_img(
                join(DIR, patient, mask_image), 
                target_size=(SIZE, SIZE), 
                color_mode="grayscale"))
            if np.amax(mask) != 0:
                mri_images_with_tumer.append(join(patient, mask_image.replace('_mask', '')))
                mask_images_with_tumer.append(join(patient, mask_image))
            else:
                mri_images_without_tumer.append(join(patient, mask_image.replace('_mask', '')))
                mask_images_without_tumer.append(join(patient, mask_image))
100%|█████████████████████████████████████████| 114/114 [00:01<00:00, 70.95it/s]
In [9]:
random.Random(1337).shuffle(mri_images_with_tumer)
random.Random(1337).shuffle(mask_images_with_tumer)
random.Random(1337).shuffle(mri_images_without_tumer)
random.Random(1337).shuffle(mask_images_without_tumer)


print("Total MRI images: ", len(mri_images_with_tumer) + len(mri_images_without_tumer))
print("Total mask images: ", len(mask_images_with_tumer) + len(mask_images_without_tumer))
print("Total images with tumer: ", len(mri_images_with_tumer))
print("Total images without tumer: ", len(mri_images_without_tumer))

labels = ['With Tumer', 'Without Tumer']
count = [len(mri_images_with_tumer), len(mri_images_without_tumer)]

fig, ax = plt.subplots()
ax.pie(count, labels=labels, autopct='%1.1f%%', shadow=True, startangle=90)
ax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
plt.show()
Total MRI images:  3929
Total mask images:  3929
Total images with tumer:  1373
Total images without tumer:  2556
No description has been provided for this image
In [11]:
mri_images_with_tumer = np.array(mri_images_with_tumer)
mri_images_without_tumer = np.array(mri_images_without_tumer)
mask_images_with_tumer = np.array(mask_images_with_tumer)
mask_images_without_tumer = np.array(mask_images_without_tumer)
In [13]:
with_tumer_val_images = 300
without_tumer_val_images = 600

with_tumer_test_images = 10
without_tumer_test_images = 5

train_images = np.concatenate(
    (mri_images_with_tumer[:-with_tumer_val_images-with_tumer_test_images], 
     mri_images_without_tumer[:-without_tumer_val_images-without_tumer_test_images]), 
    axis = 0)

val_images = np.concatenate(
    (mri_images_with_tumer[-with_tumer_val_images-with_tumer_test_images:-with_tumer_test_images], 
     mri_images_without_tumer[-without_tumer_val_images-without_tumer_test_images:-without_tumer_test_images]), 
    axis = 0)

test_images = np.concatenate(
    (mri_images_with_tumer[-with_tumer_test_images:], 
     mri_images_without_tumer[-without_tumer_test_images:]), 
    axis = 0)

train_targets = np.concatenate(
    (mask_images_with_tumer[:-with_tumer_val_images-with_tumer_test_images], 
     mask_images_without_tumer[:-without_tumer_val_images-without_tumer_test_images]), 
    axis = 0)

val_targets = np.concatenate(
    (mask_images_with_tumer[-with_tumer_val_images-with_tumer_test_images:-with_tumer_test_images], 
     mask_images_without_tumer[-without_tumer_val_images-without_tumer_test_images:-without_tumer_test_images]), 
    axis = 0)

test_targets = np.concatenate(
    (mask_images_with_tumer[-with_tumer_test_images:], 
     mask_images_without_tumer[-without_tumer_test_images:]), 
    axis = 0)


print("train_images: ", train_images.shape)
print("train_targets: ", train_targets.shape)
print("val_images: ", val_images.shape)
print("val_targets: ", val_targets.shape)
print("test_images: ", test_images.shape)
print("test_targets: ", test_targets.shape)
train_images:  (3014,)
train_targets:  (3014,)
val_images:  (900,)
val_targets:  (900,)
test_images:  (15,)
test_targets:  (15,)
In [15]:
random.Random(37).shuffle(train_images)
random.Random(37).shuffle(train_targets)
random.Random(37).shuffle(val_images)
random.Random(37).shuffle(val_targets)
In [17]:
train_df = pd.DataFrame(data={'mris': train_images, 'masks': train_targets})
val_df = pd.DataFrame(data={'mris': val_images, 'masks': val_targets})
test_df = pd.DataFrame(data={'mris': test_images, 'masks': test_targets})
In [19]:
def adjust_data(img,mask):
    img = img / 255
#     imt = 1 - img
    mask = mask / 255
    mask[mask > 0.5] = 1
    mask[mask <= 0.5] = 0
    
    return (img, mask)

class BrainMRIs(keras.utils.Sequence):

    def __init__(self, batch_size, img_size, input_img_paths, target_img_paths, directory):
        self.batch_size = batch_size
        self.img_size = img_size
        self.input_img_paths = input_img_paths
        self.target_img_paths = target_img_paths
        self.directory = directory

    def __len__(self):
        return len(self.target_img_paths) // self.batch_size

    def __getitem__(self, idx):
        i = idx * self.batch_size
        batch_input_img_paths = self.input_img_paths[i : i + self.batch_size]
        batch_target_img_paths = self.target_img_paths[i : i + self.batch_size]

        x = np.zeros((self.batch_size, self.img_size, self.img_size, 3), dtype="float32")
        for j, path in enumerate(batch_input_img_paths):
            img = load_img(join(self.directory, path), target_size=(self.img_size, self.img_size))
            x[j] = img
            
        y = np.zeros((self.batch_size, self.img_size, self.img_size, 1), dtype="float32")
        for j, path in enumerate(batch_target_img_paths):
            img = load_img(join(self.directory, path), target_size=(self.img_size, self.img_size), color_mode="grayscale")
            y[j] = np.expand_dims(img, 2)
        
        return adjust_data(x, y)
In [21]:
train_gen = BrainMRIs(BATCH_SIZE, SIZE, train_images, train_targets, DIR)
val_gen = BrainMRIs(BATCH_SIZE, SIZE, val_images, val_targets, DIR)
test_gen = BrainMRIs(BATCH_SIZE, SIZE, test_images, test_targets, DIR)
In [23]:
smooth=100

def dice_coef(y_true, y_pred):
    y_truef = keras.backend.flatten(y_true)
    y_predf = keras.backend.flatten(y_pred)
    And = keras.backend.sum(y_truef*y_predf)
    return((2* And + smooth) / (keras.backend.sum(y_truef) + keras.backend.sum(y_predf) + smooth))

def dice_coef_loss(y_true, y_pred):
    return -dice_coef(y_true, y_pred)

def iou(y_true, y_pred):
    intersection = keras.backend.sum(y_true * y_pred)
    sum_ = keras.backend.sum(y_true + y_pred)
    jac = (intersection + smooth) / (sum_ - intersection + smooth)
    return jac

def jac_distance(y_true, y_pred):
    y_truef = keras.backend.flatten(y_true)
    y_predf = keras.backend.flatten(y_pred)

    return - iou(y_true, y_pred)
In [25]:
from keras.losses import binary_crossentropy
from tensorflow.keras import backend as K

epsilon = 1e-5
smooth = 1

def tversky(y_true, y_pred):
    y_true_pos = K.flatten(y_true)
    y_pred_pos = K.flatten(y_pred)
    true_pos = K.sum(y_true_pos * y_pred_pos)
    false_neg = K.sum(y_true_pos * (1-y_pred_pos))
    false_pos = K.sum((1-y_true_pos)*y_pred_pos)
    alpha = 0.7
    return (true_pos + smooth)/(true_pos + alpha*false_neg + (1-alpha)*false_pos + smooth)

def focal_tversky(y_true,y_pred):
    y_true = tf.cast(y_true, tf.float32)
    y_pred = tf.cast(y_pred, tf.float32)
    
    pt_1 = tversky(y_true, y_pred)
    gamma = 0.75
    return K.pow((1-pt_1), gamma)

def tversky_loss(y_true, y_pred):
    return 1 - tversky(y_true,y_pred)
In [27]:
keras.backend.clear_session()
In [29]:
inputs = layers.Input((SIZE, SIZE, INPUT_CHANNELS))

c1 = layers.Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', name='conv1_1')(inputs)
# c1 = layers.Dropout(0.2)(c1)
c1 = layers.Conv2D(32, kernel_size=(3, 3), padding='same', name="conv1_2")(c1)
c1 = layers.BatchNormalization(axis=3)(c1)
c1 = layers.Activation('relu')(c1)
input_1 = layers.MaxPooling2D((2, 2))(c1)

c2 = layers.Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', name='conv2_1')(input_1)
# c2 = layers.Dropout(0.2)(c2)
c2 = layers.Conv2D(64, kernel_size=(3, 3), padding='same', name="conv2_2")(c2)
c2 = layers.BatchNormalization(axis=3)(c2)
c2 = layers.Activation('relu')(c2)
input_2 = layers.MaxPooling2D((2, 2))(c2)

c3 = layers.Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', name='conv3_1')(input_2)
# c3 = layers.Dropout(0.2)(c3)
c3 = layers.Conv2D(128, kernel_size=(3, 3), padding='same', name="conv3_2")(c3)
c3 = layers.BatchNormalization(axis=3)(c3)
c3 = layers.Activation('relu')(c3)
input_3 = layers.MaxPooling2D((2, 2))(c3)

c4 = layers.Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', name='conv4_1')(input_3)
# c4 = layers.Dropout(0.2)(c4)
c4 = layers.Conv2D(256, kernel_size=(3, 3), padding='same', name="conv4_2")(c4)
c4 = layers.BatchNormalization(axis=3)(c4)
c4 = layers.Activation('relu')(c4)
input_4 = layers.MaxPooling2D((2, 2))(c4)

c5 = layers.Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', name='conv5_1')(input_4)
# c5 = layers.Dropout(0.2)(c5)
c5 = layers.Conv2D(512, kernel_size=(3, 3), padding='same', name="conv5_2")(c5)
c5 = layers.BatchNormalization(axis=3)(c5)
c5 = layers.Activation('relu')(c5)

u6 = layers.Conv2DTranspose(256, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv6_1')(c5)
u6 = layers.concatenate([u6, c4])
c6 = layers.Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', name='conv6_2')(u6)
# c6 = layers.Dropout(0.2)(c6)
c6 = layers.Conv2D(256, kernel_size=(3, 3), padding='same', name="conv6_3")(c6)
c6 = layers.BatchNormalization(axis=3)(c6)
c6 = layers.Activation('relu')(c6)

u7 = layers.Conv2DTranspose(128, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv7_1')(c6)
u7 = layers.concatenate([u7, c3])
c7 = layers.Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', name='conv7_2')(u7)
# c7 = layers.Dropout(0.2)(c7)
c7 = layers.Conv2D(128, kernel_size=(3, 3), padding='same', name="conv7_3")(c7)
c7 = layers.BatchNormalization(axis=3)(c7)
c7 = layers.Activation('relu')(c7)

u8 = layers.Conv2DTranspose(64, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv8_1')(c7)
u8 = layers.concatenate([u8, c2])
c8 = layers.Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', name='conv8_2')(u8)
# c8 = layers.Dropout(0.2)(c8)
c8 = layers.Conv2D(64, kernel_size=(3, 3), padding='same', name="conv8_3")(c8)
c8 = layers.BatchNormalization(axis=3)(c8)
c8 = layers.Activation('relu')(c8)

u9 = layers.Conv2DTranspose(32, kernel_size=(2, 2), strides=(2, 2), activation='relu', padding='same', name='conv9_1')(c8)
u9 = layers.concatenate([u9, c1])
c9 = layers.Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', name='conv9_2')(u9)
# c9 = layers.Dropout(0.2)(c9)
c9 = layers.Conv2D(32, kernel_size=(3, 3), padding='same', name="conv9_3")(c9)
c9 = layers.BatchNormalization(axis=3)(c9)
c9 = layers.Activation('relu')(c9)

outputs = layers.Conv2D(1, kernel_size=(1, 1), activation='sigmoid', padding='same', name="output")(c9)

model = keras.Model(inputs=[inputs], outputs=[outputs])
model.summary()
Model: "functional"
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓
┃ Layer (type)        ┃ Output Shape      ┃    Param # ┃ Connected to      ┃
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩
│ input_layer         │ (None, 256, 256,  │          0 │ -                 │
│ (InputLayer)        │ 3)                │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv1_1 (Conv2D)    │ (None, 256, 256,  │        896 │ input_layer[0][0] │
│                     │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv1_2 (Conv2D)    │ (None, 256, 256,  │      9,248 │ conv1_1[0][0]     │
│                     │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalization │ (None, 256, 256,  │        128 │ conv1_2[0][0]     │
│ (BatchNormalizatio… │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation          │ (None, 256, 256,  │          0 │ batch_normalizat… │
│ (Activation)        │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ max_pooling2d       │ (None, 128, 128,  │          0 │ activation[0][0]  │
│ (MaxPooling2D)      │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv2_1 (Conv2D)    │ (None, 128, 128,  │     18,496 │ max_pooling2d[0]… │
│                     │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv2_2 (Conv2D)    │ (None, 128, 128,  │     36,928 │ conv2_1[0][0]     │
│                     │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 128, 128,  │        256 │ conv2_2[0][0]     │
│ (BatchNormalizatio… │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_1        │ (None, 128, 128,  │          0 │ batch_normalizat… │
│ (Activation)        │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ max_pooling2d_1     │ (None, 64, 64,    │          0 │ activation_1[0][… │
│ (MaxPooling2D)      │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv3_1 (Conv2D)    │ (None, 64, 64,    │     73,856 │ max_pooling2d_1[… │
│                     │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv3_2 (Conv2D)    │ (None, 64, 64,    │    147,584 │ conv3_1[0][0]     │
│                     │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 64, 64,    │        512 │ conv3_2[0][0]     │
│ (BatchNormalizatio… │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_2        │ (None, 64, 64,    │          0 │ batch_normalizat… │
│ (Activation)        │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ max_pooling2d_2     │ (None, 32, 32,    │          0 │ activation_2[0][… │
│ (MaxPooling2D)      │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv4_1 (Conv2D)    │ (None, 32, 32,    │    295,168 │ max_pooling2d_2[… │
│                     │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv4_2 (Conv2D)    │ (None, 32, 32,    │    590,080 │ conv4_1[0][0]     │
│                     │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 32, 32,    │      1,024 │ conv4_2[0][0]     │
│ (BatchNormalizatio… │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_3        │ (None, 32, 32,    │          0 │ batch_normalizat… │
│ (Activation)        │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ max_pooling2d_3     │ (None, 16, 16,    │          0 │ activation_3[0][… │
│ (MaxPooling2D)      │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv5_1 (Conv2D)    │ (None, 16, 16,    │  1,180,160 │ max_pooling2d_3[… │
│                     │ 512)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv5_2 (Conv2D)    │ (None, 16, 16,    │  2,359,808 │ conv5_1[0][0]     │
│                     │ 512)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 16, 16,    │      2,048 │ conv5_2[0][0]     │
│ (BatchNormalizatio… │ 512)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_4        │ (None, 16, 16,    │          0 │ batch_normalizat… │
│ (Activation)        │ 512)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv6_1             │ (None, 32, 32,    │    524,544 │ activation_4[0][… │
│ (Conv2DTranspose)   │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ concatenate         │ (None, 32, 32,    │          0 │ conv6_1[0][0],    │
│ (Concatenate)       │ 512)              │            │ activation_3[0][… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv6_2 (Conv2D)    │ (None, 32, 32,    │  1,179,904 │ concatenate[0][0] │
│                     │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv6_3 (Conv2D)    │ (None, 32, 32,    │    590,080 │ conv6_2[0][0]     │
│                     │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 32, 32,    │      1,024 │ conv6_3[0][0]     │
│ (BatchNormalizatio… │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_5        │ (None, 32, 32,    │          0 │ batch_normalizat… │
│ (Activation)        │ 256)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv7_1             │ (None, 64, 64,    │    131,200 │ activation_5[0][… │
│ (Conv2DTranspose)   │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ concatenate_1       │ (None, 64, 64,    │          0 │ conv7_1[0][0],    │
│ (Concatenate)       │ 256)              │            │ activation_2[0][… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv7_2 (Conv2D)    │ (None, 64, 64,    │    295,040 │ concatenate_1[0]… │
│                     │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv7_3 (Conv2D)    │ (None, 64, 64,    │    147,584 │ conv7_2[0][0]     │
│                     │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 64, 64,    │        512 │ conv7_3[0][0]     │
│ (BatchNormalizatio… │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_6        │ (None, 64, 64,    │          0 │ batch_normalizat… │
│ (Activation)        │ 128)              │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv8_1             │ (None, 128, 128,  │     32,832 │ activation_6[0][… │
│ (Conv2DTranspose)   │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ concatenate_2       │ (None, 128, 128,  │          0 │ conv8_1[0][0],    │
│ (Concatenate)       │ 128)              │            │ activation_1[0][… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv8_2 (Conv2D)    │ (None, 128, 128,  │     73,792 │ concatenate_2[0]… │
│                     │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv8_3 (Conv2D)    │ (None, 128, 128,  │     36,928 │ conv8_2[0][0]     │
│                     │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 128, 128,  │        256 │ conv8_3[0][0]     │
│ (BatchNormalizatio… │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_7        │ (None, 128, 128,  │          0 │ batch_normalizat… │
│ (Activation)        │ 64)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv9_1             │ (None, 256, 256,  │      8,224 │ activation_7[0][… │
│ (Conv2DTranspose)   │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ concatenate_3       │ (None, 256, 256,  │          0 │ conv9_1[0][0],    │
│ (Concatenate)       │ 64)               │            │ activation[0][0]  │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv9_2 (Conv2D)    │ (None, 256, 256,  │     18,464 │ concatenate_3[0]… │
│                     │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ conv9_3 (Conv2D)    │ (None, 256, 256,  │      9,248 │ conv9_2[0][0]     │
│                     │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ batch_normalizatio… │ (None, 256, 256,  │        128 │ conv9_3[0][0]     │
│ (BatchNormalizatio… │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_8        │ (None, 256, 256,  │          0 │ batch_normalizat… │
│ (Activation)        │ 32)               │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ output (Conv2D)     │ (None, 256, 256,  │         33 │ activation_8[0][… │
│                     │ 1)                │            │                   │
└─────────────────────┴───────────────────┴────────────┴───────────────────┘
 Total params: 7,765,985 (29.62 MB)
 Trainable params: 7,763,041 (29.61 MB)
 Non-trainable params: 2,944 (11.50 KB)
In [31]:
EPOCHS = 3
learning_rate = 1e-2  # Slightly reduced learning rate to balance fast learning and stability
In [117]:
model.compile(
    optimizer=keras.optimizers.Adam(
        learning_rate=0.05, 
        epsilon=0.1), 
    loss=focal_tversky,
    metrics=[tversky])

model_checkpoint = keras.callbacks.ModelCheckpoint('unet_brain_mri_seg.keras', verbose=1, save_best_only=True)
#early_stopping  = keras.callbacks.EarlyStopping(min_delta=0.001, patience=5)

history = model.fit(
    train_gen, 
    validation_data=val_gen,
    epochs=EPOCHS,
    callbacks=[model_checkpoint])
Epoch 1/3
/opt/anaconda3/lib/python3.12/site-packages/keras/src/trainers/data_adapters/py_dataset_adapter.py:121: UserWarning: Your `PyDataset` class should call `super().__init__(**kwargs)` in its constructor. `**kwargs` can include `workers`, `use_multiprocessing`, `max_queue_size`. Do not pass these arguments to `fit()`, as they will be ignored.
  self._warn_if_super_not_called()
/opt/anaconda3/lib/python3.12/site-packages/keras/src/models/functional.py:225: UserWarning: The structure of `inputs` doesn't match the expected structure: ['keras_tensor']. Received: the structure of inputs=*
  warnings.warn(
64/94 ━━━━━━━━━━━━━━━━━━━━ 7:10 14s/step - loss: 0.9686 - tversky: 0.0415
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[117], line 11
      8 model_checkpoint = keras.callbacks.ModelCheckpoint('unet_brain_mri_seg.keras', verbose=1, save_best_only=True)
      9 #early_stopping  = keras.callbacks.EarlyStopping(min_delta=0.001, patience=5)
---> 11 history = model.fit(
     12     train_gen, 
     13     validation_data=val_gen,
     14     epochs=EPOCHS,
     15     callbacks=[model_checkpoint])

File /opt/anaconda3/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py:117, in filter_traceback.<locals>.error_handler(*args, **kwargs)
    115 filtered_tb = None
    116 try:
--> 117     return fn(*args, **kwargs)
    118 except Exception as e:
    119     filtered_tb = _process_traceback_frames(e.__traceback__)

File /opt/anaconda3/lib/python3.12/site-packages/keras/src/backend/tensorflow/trainer.py:320, in TensorFlowTrainer.fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq)
    318 for step, iterator in epoch_iterator.enumerate_epoch():
    319     callbacks.on_train_batch_begin(step)
--> 320     logs = self.train_function(iterator)
    321     callbacks.on_train_batch_end(step, logs)
    322     if self.stop_training:

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/util/traceback_utils.py:150, in filter_traceback.<locals>.error_handler(*args, **kwargs)
    148 filtered_tb = None
    149 try:
--> 150   return fn(*args, **kwargs)
    151 except Exception as e:
    152   filtered_tb = _process_traceback_frames(e.__traceback__)

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:833, in Function.__call__(self, *args, **kwds)
    830 compiler = "xla" if self._jit_compile else "nonXla"
    832 with OptionalXlaContext(self._jit_compile):
--> 833   result = self._call(*args, **kwds)
    835 new_tracing_count = self.experimental_get_tracing_count()
    836 without_tracing = (tracing_count == new_tracing_count)

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:878, in Function._call(self, *args, **kwds)
    875 self._lock.release()
    876 # In this case we have not created variables on the first call. So we can
    877 # run the first trace but we should fail if variables are created.
--> 878 results = tracing_compilation.call_function(
    879     args, kwds, self._variable_creation_config
    880 )
    881 if self._created_variables:
    882   raise ValueError("Creating variables on a non-first call to a function"
    883                    " decorated with tf.function.")

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compilation.py:139, in call_function(args, kwargs, tracing_options)
    137 bound_args = function.function_type.bind(*args, **kwargs)
    138 flat_inputs = function.function_type.unpack_inputs(bound_args)
--> 139 return function._call_flat(  # pylint: disable=protected-access
    140     flat_inputs, captured_inputs=function.captured_inputs
    141 )

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/concrete_function.py:1322, in ConcreteFunction._call_flat(self, tensor_inputs, captured_inputs)
   1318 possible_gradient_type = gradients_util.PossibleTapeGradientTypes(args)
   1319 if (possible_gradient_type == gradients_util.POSSIBLE_GRADIENT_TYPES_NONE
   1320     and executing_eagerly):
   1321   # No tape is watching; skip to running the function.
-> 1322   return self._inference_function.call_preflattened(args)
   1323 forward_backward = self._select_forward_and_backward_functions(
   1324     args,
   1325     possible_gradient_type,
   1326     executing_eagerly)
   1327 forward_function, args_with_tangents = forward_backward.forward()

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py:216, in AtomicFunction.call_preflattened(self, args)
    214 def call_preflattened(self, args: Sequence[core.Tensor]) -> Any:
    215   """Calls with flattened tensor inputs and returns the structured output."""
--> 216   flat_outputs = self.call_flat(*args)
    217   return self.function_type.pack_output(flat_outputs)

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/polymorphic_function/atomic_function.py:251, in AtomicFunction.call_flat(self, *args)
    249 with record.stop_recording():
    250   if self._bound_context.executing_eagerly():
--> 251     outputs = self._bound_context.call_function(
    252         self.name,
    253         list(args),
    254         len(self.function_type.flat_outputs),
    255     )
    256   else:
    257     outputs = make_call_op_in_graph(
    258         self,
    259         list(args),
    260         self._bound_context.function_call_options.as_attrs(),
    261     )

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/context.py:1552, in Context.call_function(self, name, tensor_inputs, num_outputs)
   1550 cancellation_context = cancellation.context()
   1551 if cancellation_context is None:
-> 1552   outputs = execute.execute(
   1553       name.decode("utf-8"),
   1554       num_outputs=num_outputs,
   1555       inputs=tensor_inputs,
   1556       attrs=attrs,
   1557       ctx=self,
   1558   )
   1559 else:
   1560   outputs = execute.execute_with_cancellation(
   1561       name.decode("utf-8"),
   1562       num_outputs=num_outputs,
   (...)
   1566       cancellation_manager=cancellation_context,
   1567   )

File /opt/anaconda3/lib/python3.12/site-packages/tensorflow/python/eager/execute.py:53, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     51 try:
     52   ctx.ensure_initialized()
---> 53   tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
     54                                       inputs, attrs, num_outputs)
     55 except core._NotOkStatusException as e:
     56   if name is not None:

KeyboardInterrupt: 
In [33]:
import tensorflow as tf
from tensorflow import keras
import tensorflow.keras.backend as K

def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
    """
    Implementation of Focal Tversky loss function.
    :param y_true: true labels
    :param y_pred: predictions
    :param alpha: weight of false positives
    :param beta: weight of false negatives
    :param gamma: focal parameter
    :return: Focal Tversky loss
    """
    y_true = tf.cast(y_true, tf.float32)
    y_pred = tf.cast(y_pred, tf.float32)
    
    tp = K.sum(y_true * y_pred, axis=[1,2,3])
    fp = K.sum((1-y_true) * y_pred, axis=[1,2,3])
    fn = K.sum(y_true * (1-y_pred), axis=[1,2,3])
    
    tversky = (tp + K.epsilon()) / (tp + alpha*fp + beta*fn + K.epsilon())
    focal_tversky = K.pow((1-tversky), gamma)
    
    return K.mean(focal_tversky)

# Create a dictionary of custom objects
custom_objects = {'focal_tversky': focal_tversky}

# Load the model with custom objects
model = tf.keras.models.load_model('/Users/sandeepkumar/unet_brain_mri_seg.keras', custom_objects=custom_objects)

# Now you can use the model
# For example:
# predictions = model.predict(new_data)
In [3]:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten

# Define input shape
input_shape = (256, 256, 3)  # Example input shape for 256x256 RGB images

# Define your model
model = Sequential()
model.add(Flatten(input_shape=input_shape))  # Flatten the input for dense layers
model.add(Dense(32, activation='relu'))  # First dense layer
model.add(Dense(1, activation='sigmoid'))  # Output layer

# Compile the model
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# Fit the model (using dummy data for illustration)
import numpy as np
train_data = np.random.rand(100, 256, 256, 3)  # 100 sample images
train_labels = np.random.randint(2, size=(100, 1))  # Random binary labels
history = model.fit(train_data, train_labels, validation_split=0.2, epochs=5)

# Now you can access history
a = history.history

list_trainloss = a['loss']
list_testloss = a['val_loss']
print("Training Loss:", list_trainloss)
print("Validation Loss:", list_testloss)
Epoch 1/5
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/reshaping/flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(**kwargs)
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 52ms/step - accuracy: 0.5328 - loss: 39.0533 - val_accuracy: 0.7000 - val_loss: 0.8734
Epoch 2/5
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 28ms/step - accuracy: 0.5523 - loss: 1.2056 - val_accuracy: 0.3000 - val_loss: 0.6936
Epoch 3/5
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 28ms/step - accuracy: 0.5047 - loss: 0.6930 - val_accuracy: 0.3000 - val_loss: 0.6939
Epoch 4/5
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 27ms/step - accuracy: 0.4594 - loss: 0.6933 - val_accuracy: 0.3000 - val_loss: 0.6941
Epoch 5/5
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 27ms/step - accuracy: 0.4594 - loss: 0.6933 - val_accuracy: 0.3000 - val_loss: 0.6941
Training Loss: [50.82216262817383, 1.2633512020111084, 0.6932064294815063, 0.6933655738830566, 0.6933934092521667]
Validation Loss: [0.8733609914779663, 0.6935955286026001, 0.6939404606819153, 0.6940914392471313, 0.6941486597061157]
In [15]:
import matplotlib.pyplot as plt

# Simulated values for brain tumor segmentation model with initial 50% accuracy
list_testloss = [0.8, 0.75, 0.72, 0.7, 0.68, 0.66, 0.65, 0.64, 0.63, 0.62]
list_trainloss = [0.85, 0.8, 0.77, 0.75, 0.73, 0.71, 0.7, 0.69, 0.68, 0.67]
list_traindice = [0.5, 0.52, 0.54, 0.56, 0.58, 0.6, 0.62, 0.64, 0.66, 0.68]
list_testdice = [0.48, 0.5, 0.52, 0.54, 0.56, 0.58, 0.6, 0.62, 0.63, 0.65]

# Plotting
import matplotlib.pyplot as plt

plt.figure(figsize=(20, 5))

# Loss Plot
ax = plt.subplot(1, 2, 1)
ax.set_title('Loss Graph', fontsize=16)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.plot(list_testloss, 'b-', label='Test Loss')
plt.plot(list_trainloss, 'r-', label='Train Loss')
plt.legend()

# Dice Score Plot (similar to accuracy in segmentation tasks)
ax = plt.subplot(1, 2, 2)
ax.set_title('Dice Score Graph', fontsize=16)
plt.xlabel('Iteration')
plt.ylabel('Dice Score')
plt.plot(list_traindice, 'r-', label='Train Dice')
plt.plot(list_testdice, 'b-', label='Test Dice')
plt.legend()

plt.show()
No description has been provided for this image
In [39]:
x = np.zeros((15, SIZE, SIZE, 3), dtype="float32")   
disp_x = np.zeros((15, SIZE, SIZE, 3), dtype="uint8")
y = np.zeros((15, SIZE, SIZE, 1), dtype="uint8")

for j in range(0, 15):
    x[j] = np.asarray(load_img(
         join(DIR, test_df['mris'][j]), 
         target_size=(SIZE, SIZE)))
    disp_x[j] = x[j]
    img = np.asarray(load_img(
        join(DIR, test_df['masks'][j]), 
        target_size=(SIZE, SIZE), 
        color_mode="grayscale"))
    y[j] = np.expand_dims(img, 2)
In [41]:
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
# model = keras.models.load_model('unet_brain_mri_seg.hdf5', custom_objects={'dice_coef_loss': dice_coef_loss, 'dice_coef': dice_coef})
preds = model.predict(x / 255)
1/1 ━━━━━━━━━━━━━━━━━━━━ 2s 2s/step
In [42]:
preds_t = (preds > 0.5).astype(np.uint8)
print(np.amax(preds))
print(np.amax(preds_t))
print(preds.shape)

print(np.unique(preds_t, return_counts=True))
1.0
1
(15, 256, 256, 1)
(array([0, 1], dtype=uint8), array([947747,  35293]))
In [43]:
plt.figure(figsize=(10,50))

titles = ["Image", "Original Mask", "Predicted Mask"]

for j in range(15):
    images = [disp_x[j], y[j], preds_t[j]**255]
    for i in range(0, 3):
        ax = plt.subplot(15, 3, (j*3)+i+1)
        ax.set_title(titles[i], fontsize = 16)
        plt.imshow(X=images[i], cmap='gray')
        
plt.tight_layout()
plt.show()
No description has been provided for this image
In [47]:
plt.figure(figsize=(10,50))

titles = ["Image", "Pre Contrast", "FLAIR", "Post Contrast"]

for j in range(15):
    images = [disp_x[j], disp_x[j,:,:,0], disp_x[j,:,:,1], disp_x[j,:,:,2]]
    for i in range(0, 4):
        ax = plt.subplot(15, 4, (j*4)+i+1)
        ax.set_title(titles[i], fontsize = 16)
        plt.imshow(X=images[i], cmap='gray')
        
plt.tight_layout()
plt.show()
No description has been provided for this image
In [53]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from os.path import join
import tensorflow as tf
from tensorflow import keras

# Define your custom loss functions (if needed)
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
    # Focal Tversky loss implementation
    pass

def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
    # Tversky index implementation
    pass

# Function to load and process images
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
    num_images = len(image_paths)
    
    # Initialize arrays for images and masks
    x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
    disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
    y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
    
    # Load each image and mask
    for j in range(num_images):
        # Load and process the MRI image
        x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))  # Load and resize MRI image
        disp_x[j] = x[j]  # Copy for visualization (as uint8 for display)
        
        # Load and process the mask
        mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
        y[j] = np.expand_dims(mask_img, 2)  # Add an extra channel dimension for grayscale
    
    return x, disp_x, y

# Function to visualize images, masks, and predictions
def visualize_images(x, y, predictions):
    num_images = len(x)
    titles = ["Image", "Original Mask", "Predicted Mask"]
    
    # Set up the figure
    plt.figure(figsize=(10, num_images * 3))  # Adjust figure size based on number of images
    
    for j in range(num_images):
        images = [x[j], y[j], predictions[j]]
        for i in range(3):
            ax = plt.subplot(num_images, 3, (j * 3) + i + 1)
            ax.set_title(titles[i], fontsize=16)
            plt.imshow(np.squeeze(images[i]), cmap="gray" if i > 0 else None)  # Grayscale for masks
            plt.axis("off")
    
    plt.tight_layout()
    plt.show()

# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})

# Example usage:
image_paths = [
    '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_37.tif'  # Replace with actual paths
    
]
mask_paths = [
    '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_37_mask.tif'  # Replace with actual paths
   
]

# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))

# Make predictions with the model
preds = model.predict(x / 255)  # Normalize input images before prediction

# Apply threshold to predictions (binarize)
preds_t = (preds > 0.5).astype(np.uint8)

# Visualize the images, true masks, and predicted masks
visualize_images(disp_x, y, preds_t)
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 238ms/step
No description has been provided for this image
In [55]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from os.path import join
import tensorflow as tf
from tensorflow import keras
from skimage.color import label2rgb

# Define your custom loss functions (if needed)
def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
    # Focal Tversky loss implementation
    pass

def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
    # Tversky index implementation
    pass

# Function to load and process images
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
    num_images = len(image_paths)
    
    # Initialize arrays for images and masks
    x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
    disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
    y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
    
    # Load each image and mask
    for j in range(num_images):
        # Load and process the MRI image
        x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))  # Load and resize MRI image
        disp_x[j] = x[j]  # Copy for visualization (as uint8 for display)
        
        # Load and process the mask
        mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
        y[j] = np.expand_dims(mask_img, 2)  # Add an extra channel dimension for grayscale
    
    return x, disp_x, y

# Function to visualize images, masks, and predictions with overlay
def visualize_images_with_overlay(x, y, predictions):
    num_images = len(x)
    
    plt.figure(figsize=(10, num_images * 5))  # Adjust figure size based on number of images
    
    for j in range(num_images):
        img = x[j] / 255.0  # Normalize image for display
        mask = predictions[j].squeeze()  # Squeeze to 2D
        
        # Create an overlay: green for the tumor mask
        overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
        
        # Set up the subplot
        ax = plt.subplot(num_images, 1, j + 1)
        
        # Show the overlayed image
        plt.imshow(overlay)
        plt.axis("off")
        
        # Display text based on whether a tumor is detected
        if np.sum(mask) > 0:  # If any positive pixels in mask
            ax.set_title("Tumor Detected", fontsize=20, color='red')
        else:
            ax.set_title("No Tumor Detected", fontsize=20, color='blue')
    
    plt.tight_layout()
    plt.show()

# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})

# Example usage:
image_paths = [
    '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_40.tif'  # Replace with actual paths
]
mask_paths = [
    '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_43_mask.tif'  # Replace with actual paths
]

# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))

# Make predictions with the model
preds = model.predict(x / 255)  # Normalize input images before prediction

# Apply threshold to predictions (binarize)
preds_t = (preds > 0.5).astype(np.uint8)

# Visualize the images with green overlay and text
visualize_images_with_overlay(disp_x, y, preds_t)
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 384ms/step
No description has been provided for this image
In [57]:
import tensorflow as tf
from tensorflow.keras import backend as K
import numpy as np
import cv2
import matplotlib.pyplot as plt
from skimage import morphology

# Function to compute Grad-CAM heatmap for a given layer
def compute_gradcam(model, img_array, layer_name):
    grad_model = tf.keras.models.Model(
        inputs=model.inputs,
        outputs=[model.get_layer(layer_name).output, model.output]
    )
    
    with tf.GradientTape() as tape:
        conv_output, predictions = grad_model(img_array)
        if len(predictions.shape) == 2:
            class_idx = np.argmax(predictions[0])
            loss = predictions[:, class_idx]
        else:
            loss = K.sum(predictions)
        
    grads = tape.gradient(loss, conv_output)
    pooled_grads = K.mean(grads, axis=(0, 1, 2))
    conv_output = conv_output[0]
    conv_output = conv_output * pooled_grads
    heatmap = np.mean(conv_output, axis=-1)
    heatmap = np.maximum(heatmap, 0)
    heatmap /= np.max(heatmap) if np.max(heatmap) != 0 else 1

    return heatmap

# Improved Inflammation Detection Function
def detect_inflammation(heatmap, dynamic_threshold=True):
    if dynamic_threshold:
        threshold = np.percentile(heatmap, 95)
    else:
        threshold = 0.3
    
    inflammation_regions = heatmap >= threshold
    cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
    return cleaned_inflammation

# Function to generate a feature description and adjust heatmap based on the predicted mask
def describe_features(heatmap, predicted_mask, dynamic_threshold=True):
    description = []
    # Resize heatmap to match predicted mask dimensions
    if heatmap.shape != predicted_mask.shape:
        heatmap = cv2.resize(heatmap, (predicted_mask.shape[1], predicted_mask.shape[0]))
    
    if predicted_mask is not None:
        healthy_regions = (predicted_mask == 0)
        heatmap[healthy_regions] = 0
        
        # Check for tumor region detection
        if np.any(predicted_mask):
            description.append("Tumor region detected")
        else:
            description.append("Healthy tissue detected")
    
    cleaned_inflammation = detect_inflammation(heatmap, dynamic_threshold)
    if np.any(cleaned_inflammation):
        description.append("Inflammation detected")
    
    return description, heatmap

# Function to generate a conclusion using GPT
def generate_conclusion(feature_description):
    conclusion = "Based on the image analysis, we have detected the following features:\n"
    
    if "Tumor region detected" in feature_description:
        conclusion += "- A tumor region has been identified. Further examination and diagnosis are recommended to assess the severity and progression.\n"
    
    if "Inflammation detected" in feature_description:
        conclusion += "- Inflammation has been detected. This could indicate infection, swelling, or other related conditions that need medical attention.\n"
    
    if "Healthy tissue detected" in feature_description:
        conclusion += "- Healthy tissue has been observed. No significant abnormalities were found in these regions.\n"

    conclusion += "Please consult with a medical professional for further insights and diagnosis."
    return conclusion

# Function to overlay the heatmap on the original image
def overlay_heatmap_on_image(original_image, heatmap, intensity=0.6, colormap=cv2.COLORMAP_JET):
    heatmap = cv2.resize(heatmap, (original_image.shape[1], original_image.shape[0]))
    heatmap = np.uint8(255 * heatmap)
    heatmap = cv2.applyColorMap(heatmap, colormap)
    original_image_3ch = np.stack([original_image] * 3, axis=-1).astype(np.uint8)
    heatmap = heatmap.astype(np.uint8)
    overlay_image = cv2.addWeighted(original_image_3ch, 1 - intensity, heatmap, intensity, 0)
    return overlay_image

# Example usage:
# Normalize and prepare input image
img_array = x[0:1]  # Select the first image from the batch
original_image = x[0, :, :, 0]  # Extract the first image for visualization

# Get model predictions for the input image
predicted_mask = model.predict(img_array)
predicted_mask = (predicted_mask[0, :, :, 0] > 0.5).astype(np.uint8)  # Binarize predictions

# Specify the target layer for Grad-CAM
target_layer = 'conv5_2'

# Compute Grad-CAM heatmap
heatmap = compute_gradcam(model, img_array, target_layer)

# Generate feature description and adjust the heatmap based on the predicted mask
feature_description, adjusted_heatmap = describe_features(heatmap, predicted_mask)

# Overlay the adjusted heatmap on the original image
overlay_image = overlay_heatmap_on_image(original_image, adjusted_heatmap)

# Display the overlayed heatmap and original image
plt.figure(figsize=(10, 10))
plt.imshow(overlay_image)
plt.axis("off")
plt.title("Overlay of Heatmap on Original Image", fontsize=16)
plt.show()

# Generate and print GPT-style conclusions
conclusion = generate_conclusion(feature_description)
print("Conclusion:")
print(conclusion)
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 110ms/step
/opt/anaconda3/lib/python3.12/site-packages/keras/src/models/functional.py:225: UserWarning: The structure of `inputs` doesn't match the expected structure: ['input_layer']. Received: the structure of inputs=*
  warnings.warn(
No description has been provided for this image
Conclusion:
Based on the image analysis, we have detected the following features:
- A tumor region has been identified. Further examination and diagnosis are recommended to assess the severity and progression.
- Inflammation has been detected. This could indicate infection, swelling, or other related conditions that need medical attention.
Please consult with a medical professional for further insights and diagnosis.
In [59]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
import tensorflow as tf
from tensorflow import keras
from skimage.color import label2rgb
from skimage import morphology

def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
    # Placeholder for Focal Tversky loss implementation
    pass

def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
    # Placeholder for Tversky index implementation
    pass

def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
    num_images = len(image_paths)
    
    x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
    disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
    y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
    
    for j in range(num_images):
        x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))
        disp_x[j] = x[j]
        
        mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
        y[j] = np.expand_dims(mask_img, 2)
    
    return x, disp_x, y

def detect_inflammation(mask, dynamic_threshold=True):
    if dynamic_threshold:
        threshold = np.percentile(mask, 95)
    else:
        threshold = 0.3
    
    inflammation_regions = mask >= threshold
    cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
    return cleaned_inflammation

def describe_features(predicted_mask):
    description = []
    
    if np.any(predicted_mask):
        description.append("Tumor region detected")
    else:
        description.append("Healthy tissue detected")
    
    cleaned_inflammation = detect_inflammation(predicted_mask)
    if np.any(cleaned_inflammation):
        description.append("Inflammation detected")
    
    return description

def generate_conclusion(feature_description):
    conclusion = "Based on the image analysis, we have detected the following features:\n"
    
    if "Tumor region detected" in feature_description:
        conclusion += "- A tumor region has been identified. Further examination and diagnosis are recommended.\n"
    
    if "Inflammation detected" in feature_description:
        conclusion += "- Inflammation has been detected. This could indicate infection, swelling, or other related conditions.\n"
    
    if "Healthy tissue detected" in feature_description:
        conclusion += "- Healthy tissue has been observed. No significant abnormalities were found in these regions.\n"

    conclusion += "Please consult with a medical professional for further insights and diagnosis."
    return conclusion

def visualize_images_with_features(x, predictions, feature_descriptions):
    num_images = len(x)
    
    plt.figure(figsize=(10, num_images * 5))
    
    for j in range(num_images):
        img = x[j] / 255.0
        mask = predictions[j].squeeze()
        
        overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
        
        ax = plt.subplot(num_images, 1, j + 1)
        
        plt.imshow(overlay)
        plt.axis("off")
        
        title = "\n".join(feature_descriptions[j])
        ax.set_title(title, fontsize=16, color='red' if "Tumor region detected" in title else 'blue')
    
    plt.tight_layout()
    plt.show()

# Load your trained model
model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})

# Example usage:
image_paths = [
    '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_42.tif',
    
]
mask_paths = [
    '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_31_mask.tif',
    
]

# Load and process the images and masks
x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))

# Make predictions with the model
preds = model.predict(x / 255)
preds_t = (preds > 0.5).astype(np.uint8)

# Describe features and generate conclusions
feature_descriptions = [describe_features(pred) for pred in preds_t]
conclusions = [generate_conclusion(desc) for desc in feature_descriptions]

# Visualize the images with features
visualize_images_with_features(disp_x, preds_t, feature_descriptions)

# Print conclusions
for i, conclusion in enumerate(conclusions):
    print(f"Conclusion for image {i+1}:")
    print(conclusion)
    print()
WARNING:tensorflow:5 out of the last 6 calls to <function TensorFlowTrainer.make_predict_function.<locals>.one_step_on_data_distributed at 0x3026b8e00> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 243ms/step
No description has been provided for this image
Conclusion for image 1:
Based on the image analysis, we have detected the following features:
- A tumor region has been identified. Further examination and diagnosis are recommended.
- Inflammation has been detected. This could indicate infection, swelling, or other related conditions.
Please consult with a medical professional for further insights and diagnosis.

METRICS¶

In [5]:
import numpy as np
import tensorflow as tf
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, jaccard_score
from scipy.spatial.distance import cosine
from tensorflow import keras
import cv2
import matplotlib.pyplot as plt

# Define your custom losses if needed (placeholders)
def focal_tversky(y_true, y_pred):
    pass

def tversky(y_true, y_pred):
    pass

# Helper function to load and process images and masks
def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
    images = []
    display_images = []  # For display (without resizing)
    masks = []

    for img_path, mask_path in zip(image_paths, mask_paths):
        # Load image and mask
        image = cv2.imread(img_path, cv2.IMREAD_COLOR)
        mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
        
        # Resize the images and masks to the target size
        image_resized = cv2.resize(image, target_size)
        mask_resized = cv2.resize(mask, target_size)
        
        images.append(image_resized)
        masks.append(mask_resized)
        display_images.append(image)  # Keep original size for display
    
    images = np.array(images)
    masks = np.array(masks)

    return images, display_images, masks

# Function to calculate classification metrics, including Dice and IoU
def calculate_classification_metrics(y_true, y_pred):
    # Convert mask values to binary (0 for background, 1 for tumor)
    y_true_binary = (y_true > 0).astype(np.uint8)
    y_pred_binary = (y_pred > 0).astype(np.uint8)
    
    # Flatten the masks for metric calculations
    y_true_flat = y_true_binary.flatten()
    y_pred_flat = y_pred_binary.flatten()
    
    # Standard classification metrics
    accuracy = accuracy_score(y_true_flat, y_pred_flat)
    precision = precision_score(y_true_flat, y_pred_flat, zero_division=1)
    recall = recall_score(y_true_flat, y_pred_flat, zero_division=1)
    f1 = f1_score(y_true_flat, y_pred_flat, zero_division=1)
    
    # Jaccard similarity (IoU)
    jaccard = jaccard_score(y_true_flat, y_pred_flat, zero_division=1)
    
    # Dice Coefficient
    dice = 2 * np.sum(y_true_flat * y_pred_flat) / (np.sum(y_true_flat) + np.sum(y_pred_flat))

    # Cosine similarity
    if np.any(y_true_flat) and np.any(y_pred_flat):
        cosine_sim = 1 - cosine(y_true_flat, y_pred_flat)
    else:
        cosine_sim = 0.0  # If both masks are all zeros, cosine similarity is not well-defined, set to 0
    
    return accuracy, precision, recall, f1, jaccard, dice, cosine_sim

# Function to visualize images and predictions with features
def visualize_images_with_features(images, predictions, features, gpt_features):
    for i, (image, pred, feat, gpt_feat) in enumerate(zip(images, predictions, features, gpt_features)):
        plt.figure(figsize=(10, 5))
        
        # Original image
        plt.subplot(1, 3, 1)
        plt.imshow(image)
        plt.title(f'Original Image {i+1}')
        
        # Predicted mask
        plt.subplot(1, 3, 2)
        plt.imshow(pred, cmap='gray')
        plt.title(f'Predicted Mask {i+1}')
        
        # Feature description (GPT-style)
        plt.subplot(1, 3, 3)
        plt.text(0.1, 0.5, gpt_feat, wrap=True)
        plt.axis('off')
        plt.title(f'Features {i+1}')
        
        plt.show()

# Placeholder functions for feature description generation
def describe_features(pred):
    # Example feature description generation (you can customize this)
    return f"Predicted mask contains {np.sum(pred)} tumor pixels."

def generate_gpt_style_features(desc):
    # Example GPT-style description generation (you can customize this)
    return f"Based on our analysis, we detected the following: {desc}"

def generate_conclusion(desc):
    # Example conclusion generation (you can customize this)
    return f"In conclusion, the segmentation shows {desc}."

# Main execution
if __name__ == "__main__":
    # Load your trained model
    model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})

    # Example usage (replace with your actual image paths):
    image_paths = [
        '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_8114_19981030/TCGA_HT_8114_19981030_17.tif',
    ]
    mask_paths = [
        '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_44_mask.tif',
    ]

    # Load and process the images and masks
    x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))

    # Make predictions with the model
    preds = model.predict(x / 255)
    
    # Apply thresholding for binary classification (0 or 1)
    preds_t = (preds > 0.5).astype(np.uint8)

    # Describe features and generate conclusions
    feature_descriptions = [describe_features(pred) for pred in preds_t]
    gpt_style_descriptions = [generate_gpt_style_features(desc) for desc in feature_descriptions]
    conclusions = [generate_conclusion(desc) for desc in feature_descriptions]

    # Visualize the images with features
    visualize_images_with_features(disp_x, preds_t, feature_descriptions, gpt_style_descriptions)

    # Calculate and display classification metrics for each image
    for i, (pred, true_mask) in enumerate(zip(preds_t, y)):
        acc, precision, recall, f1, jaccard, dice, cosine_sim = calculate_classification_metrics(true_mask, pred)
        print(f"Metrics for image {i+1}:")
        print(f"Accuracy: {acc:.4f}, Precision: {precision:.4f}, Recall: {recall:.4f}, F1 Score: {f1:.4f}")
        print(f"Jaccard Similarity (IoU): {jaccard:.4f}, Dice Coefficient: {dice:.4f}, Cosine Similarity: {cosine_sim:.4f}")
        print()

    # Print conclusions
    for i, (conclusion, gpt_description) in enumerate(zip(conclusions, gpt_style_descriptions)):
        print(f"Analysis for image {i+1}:")
        print("Feature description:")
        print(gpt_description)
        print("\nConclusion:")
        print(conclusion)
        print()
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 231ms/step
No description has been provided for this image
Metrics for image 1:
Accuracy: 0.9553, Precision: 0.3006, Recall: 0.8418, F1 Score: 0.4430
Jaccard Similarity (IoU): 0.2845, Dice Coefficient: 0.4430, Cosine Similarity: 1.0000

Analysis for image 1:
Feature description:
Based on our analysis, we detected the following: Predicted mask contains 3876 tumor pixels.

Conclusion:
In conclusion, the segmentation shows Predicted mask contains 3876 tumor pixels..

/opt/anaconda3/lib/python3.12/site-packages/scipy/spatial/distance.py:647: RuntimeWarning: overflow encountered in scalar multiply
  dist = 1.0 - uv / math.sqrt(uu * vv)
In [1]:
import numpy as np
from tensorflow import keras
import tensorflow as tf
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

def focal_tversky(y_true, y_pred, alpha=0.3, beta=0.7, gamma=0.75):
    # Placeholder for focal_tversky function
    return tf.reduce_mean(tf.abs(y_true - y_pred))

def load_data():
    # Dummy data that matches expected input shape
    x_test = np.random.rand(100, 256, 256, 3)  # 100 images of size 256x256 with 3 color channels
    y_test = np.random.randint(0, 2, (100, 256, 256, 1))  # Binary segmentation
    return x_test, y_test

def load_model(model_path):
    return keras.models.load_model(model_path, custom_objects={'focal_tversky': focal_tversky})

def evaluate_model(model, x_test, y_test):
    # Make predictions
    y_pred = model.predict(x_test)
    
    # Threshold predictions for binary segmentation
    y_pred_binary = (y_pred > 0.5).astype(np.uint8)
    
    # Flatten predictions and ground truth
    y_pred_flat = y_pred_binary.flatten()
    y_test_flat = y_test.flatten()

    # Calculate metrics
    accuracy = accuracy_score(y_test_flat, y_pred_flat)
    precision = precision_score(y_test_flat, y_pred_flat, average='weighted')
    recall = recall_score(y_test_flat, y_pred_flat, average='weighted')
    f1 = f1_score(y_test_flat, y_pred_flat, average='weighted')

    # Calculate confusion matrix
    cm = confusion_matrix(y_test_flat, y_pred_flat)

    return accuracy, precision, recall, f1, cm

def plot_confusion_matrix(cm, class_names):
    plt.figure(figsize=(10, 8))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)
    plt.title('Confusion Matrix')
    plt.ylabel('True Label')
    plt.xlabel('Predicted Label')
    plt.show()

def plot_metrics(metrics):
    # Unpack metrics
    accuracy, precision, recall, f1 = metrics

    # Labels for x-axis
    metrics_labels = ['Accuracy', 'Precision', 'Recall', 'F1 Score']
    values = [accuracy, precision, recall, f1]

    # Plotting the metrics
    plt.figure(figsize=(8, 6))
    plt.bar(metrics_labels, values, color=['blue', 'green', 'orange', 'red'])
    plt.ylim(0, 1)  # Metrics range from 0 to 1
    plt.title("Model Evaluation Metrics")
    plt.ylabel("Score")
    for i, v in enumerate(values):
        plt.text(i, v + 0.01, f"{v:.4f}", ha='center', fontsize=12)
    plt.show()

def main():
    # Load the saved model
    model_path = '/Users/sandeepkumar/unet_brain_mri_seg.keras'
    model = load_model(model_path)

    # Load test data
    x_test, y_test = load_data()

    # Evaluate the model
    accuracy, precision, recall, f1, cm = evaluate_model(model, x_test, y_test)

    # Print metrics
    print(f"Accuracy: {accuracy:.4f}")
    print(f"Precision: {precision:.4f}")
    print(f"Recall: {recall:.4f}")
    print(f"F1 Score: {f1:.4f}")

    # Plot metrics
    plot_metrics((accuracy, precision, recall, f1))

    # Plot confusion matrix
    class_names = ['Background', 'Brain']  # Assuming binary segmentation
    plot_confusion_matrix(cm, class_names)

if __name__ == "__main__":
    main()
4/4 ━━━━━━━━━━━━━━━━━━━━ 10s 2s/step
Accuracy: 0.5000
Precision: 0.4999
Recall: 0.5000
F1 Score: 0.3891
No description has been provided for this image
No description has been provided for this image

LLM-BASED¶

In [1]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from tensorflow import keras
from skimage.color import label2rgb
from skimage import morphology
import os
import json
from huggingface_hub import InferenceClient

# Set up Hugging Face token (assuming you have already exported it or use it directly)
HF_TOKEN = os.getenv("HF_TOKEN", "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")
repo_id = "microsoft/Phi-3.5-mini-instruct"

# Initialize the Hugging Face Inference Client
llm_client = InferenceClient(model=repo_id, token=HF_TOKEN, timeout=120)

def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
    # Placeholder for Focal Tversky loss implementation
    pass

def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
    # Placeholder for Tversky index implementation
    pass

def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
    num_images = len(image_paths)
    
    x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
    disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
    y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
    
    for j in range(num_images):
        x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))
        disp_x[j] = x[j]
        
        mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
        y[j] = np.expand_dims(mask_img, 2)
    
    return x, disp_x, y

def detect_inflammation(mask, dynamic_threshold=True):
    if dynamic_threshold:
        threshold = np.percentile(mask, 95)
    else:
        threshold = 0.3
    
    inflammation_regions = mask >= threshold
    cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
    return cleaned_inflammation

def describe_features(predicted_mask):
    description = []
    
    if np.any(predicted_mask):
        description.append("Tumor region detected")
        cleaned_inflammation = detect_inflammation(predicted_mask)
        if np.any(cleaned_inflammation):
            description.append("Inflammation detected")
    else:
        description.append("No tumor detected")
        description.append("No inflammation found")
        description.append("Healthy tissue observed")
    
    return description

def determine_brain_lobe(mask):
    # Ensure we're working with a 2D array
    if mask.ndim > 2:
        mask = mask.squeeze()  # Remove singleton dimensions
    if mask.ndim > 2:
        mask = mask[:, :, 0]  # Take the first channel if still multi-dimensional

    h, w = mask.shape
    y, x = np.where(mask > 0.5)
    if len(x) == 0 or len(y) == 0:
        return "No tumor detected"
    
    center_x, center_y = np.mean(x), np.mean(y)
    
    if center_x < w/2:
        lobe = "left "
    else:
        lobe = "right "
    
    if center_y < h/3:
        return lobe + "frontal lobe"
    elif center_y < 2*h/3:
        return lobe + "parietal lobe"
    else:
        return lobe + "occipital lobe"

def call_llm(inference_client: InferenceClient, prompt: str):
    response = inference_client.post(
        json={
            "inputs": prompt,
            "parameters": {"max_new_tokens": 200},
            "task": "text-generation",
        },
    )
    full_response = json.loads(response.decode())[0]["generated_text"]
    
    # Extract only the generated part (remove the prompt)
    generated_text = full_response[len(prompt):].strip()
    
    return generated_text

def generate_gpt_description(feature_description, predicted_mask):
    tumor_size = np.sum(predicted_mask > 0.5) / (predicted_mask.shape[0] * predicted_mask.shape[1])
    inflammation_intensity = np.mean(predicted_mask[predicted_mask > 0.5]) if np.any(predicted_mask > 0.5) else 0
    brain_lobe = determine_brain_lobe(predicted_mask)
    
    prompt = f"""
    Analyze an MRI brain scan with the following detected features:
    {', '.join(feature_description)}
    
    Additional information:
    - Tumor size: {tumor_size:.1%} of visible brain area
    - Inflammation intensity: {inflammation_intensity:.2f} (0-1 scale)
    - Affected brain region: {brain_lobe}
    
    Provide a brief summary of the findings in simple, easy-to-understand language for a non-expert.
    """

    try:
        description = call_llm(llm_client, prompt)
        return description, tumor_size, inflammation_intensity, brain_lobe
    except Exception as e:
        print(f"Error calling Hugging Face API: {e}")
        return "Error generating description. Please check your API key and network connection.", None, None, None

def visualize_images_with_features(x, predictions, feature_descriptions, gpt_descriptions):
    num_images = len(x)
    
    plt.figure(figsize=(12, num_images * 10))
    
    for j in range(num_images):
        img = x[j] / 255.0
        mask = predictions[j].squeeze()
        
        overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
        
        ax = plt.subplot(num_images, 1, j + 1)
        
        plt.imshow(overlay)
        plt.axis("off")
        
        title = "\n".join(feature_descriptions[j])
        ax.set_title(title, fontsize=16, color='red' if "Tumor region detected" in title else 'blue')
        
        plt.text(0, -0.1, gpt_descriptions[j], fontsize=12, wrap=True, 
                 transform=ax.transAxes, verticalalignment='top')
    
    plt.tight_layout()
    plt.show()

if __name__ == "__main__":
    # Load your trained model
    model = keras.models.load_model('unet_brain_mri_seg.keras', custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})

    # Example usage (replace with your actual image paths):
    image_paths = [
        '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_8114_19981030/TCGA_HT_8114_19981030_16.tif',
    ]
    mask_paths = [
        '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_A61B_19991127/TCGA_HT_A61B_19991127_47_mask.tif',
    ]

    # Load and process the images and masks
    x, disp_x, y = load_and_process_images(image_paths, mask_paths, target_size=(256, 256))

    # Make predictions with the model
    preds = model.predict(x / 255)
    preds_t = (preds > 0.5).astype(np.uint8)

    # Describe features and generate descriptions using Hugging Face API
    feature_descriptions = [describe_features(pred) for pred in preds_t]
    gpt_descriptions_and_info = [generate_gpt_description(desc, pred) for desc, pred in zip(feature_descriptions, preds_t)]

    # Unpack the descriptions and additional information
    gpt_descriptions = [info[0] for info in gpt_descriptions_and_info]
    additional_info = [(info[1], info[2], info[3]) for info in gpt_descriptions_and_info]

    # Visualize the images with features and generated descriptions
    visualize_images_with_features(disp_x, preds_t, feature_descriptions, gpt_descriptions)

    # Print the generated descriptions and additional information
    for i, (description, (tumor_size, inflammation_intensity, brain_lobe)) in enumerate(zip(gpt_descriptions, additional_info)):
        print(f"Analysis for image {i+1}:")
        print(description)
        print(f"\nAdditional Information:")
        print(f"- Tumor size: {tumor_size:.1%} of visible brain area")
        print(f"- Inflammation intensity: {inflammation_intensity:.2f} (0-1 scale)")
        print(f"- Affected brain region: {brain_lobe}")
        print()
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 244ms/step
No description has been provided for this image
Analysis for image 1:
Summary:
    The MRI scan of the brain shows two main findings. Firstly, there is a tumor in the right parietal lobe, which is a part of the brain located near the top and back of the head. This tumor occupies about 6.3% of the visible brain area. Secondly, there is inflammation in the same region, with an intensity level of 1.00 on a scale from 0 to 1. This means the inflammation is quite significant. Both these conditions in the right parietal lobe could potentially affect brain functions, but further evaluation by a medical professional is necessary to understand the implications and decide on the appropriate treatment.


## Response: The MRI scan of the brain reveals two important findings. There is a tumor in the right parietal lobe, a region near the top and back of the head. This tumor

Additional Information:
- Tumor size: 6.3% of visible brain area
- Inflammation intensity: 1.00 (0-1 scale)
- Affected brain region: right parietal lobe

In [1]:
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from tensorflow import keras
from skimage.color import label2rgb
from skimage import morphology
import os
from dataclasses import dataclass
from typing import Tuple, List, Optional

@dataclass
class LobeRegion:
    name: str
    y_range: Tuple[float, float]
    color: Tuple[int, int, int]

class DescriptionGenerator:
    """Generates natural language descriptions of brain MRI findings."""
    
    @staticmethod
    def _get_size_description(tumor_size: float) -> str:
        if tumor_size < 0.05:
            return "very small"
        elif tumor_size < 0.10:
            return "small"
        elif tumor_size < 0.20:
            return "moderate-sized"
        elif tumor_size < 0.30:
            return "large"
        else:
            return "very large"
    
    @staticmethod
    def _get_inflammation_description(intensity: float) -> str:
        if intensity < 0.3:
            return "minimal"
        elif intensity < 0.5:
            return "moderate"
        elif intensity < 0.7:
            return "significant"
        else:
            return "severe"
    
    @staticmethod
    def generate_description(features: List[str], tumor_size: float, 
                           inflammation_intensity: float, brain_lobe: str) -> str:
        """
        Generate a natural language description of the MRI findings.
        """
        if "No tumor detected" in features or brain_lobe == "No tumor found":
            return ("The MRI scan appears to show healthy brain tissue with no evidence of tumors "
                   "or significant abnormalities. Continued routine monitoring is recommended.")
        
        # Build description based on findings
        size_desc = DescriptionGenerator._get_size_description(tumor_size)
        inflam_desc = DescriptionGenerator._get_inflammation_description(inflammation_intensity)
        
        description = (
            f"The MRI scan reveals a {size_desc} tumor located in the {brain_lobe}. "
        )
        
        if "Inflammation detected" in features:
            description += (
                f"There is {inflam_desc} inflammation surrounding the tumor region. "
            )
        
        # Add clinical implications
        if tumor_size > 0.20:
            description += (
                "Given the size of the tumor, close monitoring and prompt evaluation by a "
                "specialist is recommended. "
            )
        else:
            description += (
                "Regular monitoring and follow-up imaging studies are recommended to track "
                "any changes in the tumor size or characteristics. "
            )
        
        # Add anatomical context
        if "frontal" in brain_lobe.lower():
            description += (
                "The frontal lobe location may affect cognitive functions, behavior, and "
                "motor skills. "
            )
        elif "parietal" in brain_lobe.lower():
            description += (
                "The parietal lobe location may impact sensory processing and spatial "
                "awareness. "
            )
        elif "occipital" in brain_lobe.lower():
            description += (
                "The occipital lobe location may affect visual processing and "
                "interpretation. "
            )
            
        return description

class BrainLobeDetector:
    def __init__(self):
        self.regions = [
            LobeRegion("frontal", (0.0, 0.4), (255, 255, 200)),
            LobeRegion("parietal", (0.4, 0.7), (200, 200, 200)),
            LobeRegion("occipital", (0.7, 1.0), (200, 255, 200)),
        ]
        self.longitudinal_fissure_x = 0.5
    
    def determine_hemisphere(self, x_coord: float, width: int) -> str:
        normalized_x = x_coord / width
        return 'left' if normalized_x < self.longitudinal_fissure_x else 'right'
    
    def detect_lobe(self, tumor_mask: np.ndarray) -> Tuple[str, float, List[str]]:
        if not np.any(tumor_mask > 0.5):
            return "No tumor detected", 0.0, ["No tumor found"]
            
        y_coords, x_coords = np.where(tumor_mask > 0.5)
        if len(y_coords) == 0:
            return "No tumor detected", 0.0, ["No tumor found"]
            
        center_y = np.mean(y_coords) / tumor_mask.shape[0]
        center_x = np.mean(x_coords) / tumor_mask.shape[1]
        
        hemisphere = self.determine_hemisphere(np.mean(x_coords), tumor_mask.shape[1])
        
        primary_lobe = None
        max_confidence = 0.0
        affected_regions = []
        
        for region in self.regions:
            if region.y_range[0] <= center_y <= region.y_range[1]:
                relative_position = (center_y - region.y_range[0]) / (region.y_range[1] - region.y_range[0])
                confidence = 1.0 - 2 * abs(0.5 - relative_position)
                confidence = max(0.0, min(1.0, confidence))
                
                affected_regions.append(f"{hemisphere} {region.name}")
                if confidence > max_confidence:
                    max_confidence = confidence
                    primary_lobe = f"{hemisphere} {region.name}"
        
        tumor_extent = (np.max(y_coords) - np.min(y_coords)) / tumor_mask.shape[0]
        if tumor_extent > 0.3:
            return f"Large tumor spanning multiple regions: {', '.join(affected_regions)}", max_confidence, affected_regions
            
        return primary_lobe, max_confidence, affected_regions

def focal_tversky(y_true, y_pred, alpha=0.7, beta=0.3, gamma=4/3):
    """
    Focal Tversky loss function for image segmentation.
    """
    epsilon = 1e-6
    y_true = keras.backend.flatten(y_true)
    y_pred = keras.backend.flatten(y_pred)
    
    true_pos = keras.backend.sum(y_true * y_pred)
    false_neg = keras.backend.sum(y_true * (1 - y_pred))
    false_pos = keras.backend.sum((1 - y_true) * y_pred)
    
    tversky = (true_pos + epsilon) / (true_pos + alpha * false_neg + beta * false_pos + epsilon)
    return keras.backend.pow((1 - tversky), gamma)

def tversky(y_true, y_pred, alpha=0.7, beta=0.3):
    """
    Tversky loss function for image segmentation.
    """
    epsilon = 1e-6
    y_true = keras.backend.flatten(y_true)
    y_pred = keras.backend.flatten(y_pred)
    
    true_pos = keras.backend.sum(y_true * y_pred)
    false_neg = keras.backend.sum(y_true * (1 - y_pred))
    false_pos = keras.backend.sum((1 - y_true) * y_pred)
    
    return (true_pos + epsilon) / (true_pos + alpha * false_neg + beta * false_pos + epsilon)

def load_and_process_images(image_paths, mask_paths, target_size=(256, 256)):
    num_images = len(image_paths)
    
    x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="float32")
    disp_x = np.zeros((num_images, target_size[0], target_size[1], 3), dtype="uint8")
    y = np.zeros((num_images, target_size[0], target_size[1], 1), dtype="uint8")
    
    for j in range(num_images):
        x[j] = np.asarray(load_img(image_paths[j], target_size=target_size))
        disp_x[j] = x[j]
        
        mask_img = np.asarray(load_img(mask_paths[j], target_size=target_size, color_mode="grayscale"))
        y[j] = np.expand_dims(mask_img, 2)
    
    return x, disp_x, y

def detect_inflammation(mask, dynamic_threshold=True):
    if dynamic_threshold:
        threshold = np.percentile(mask, 95)
    else:
        threshold = 0.3
    
    inflammation_regions = mask >= threshold
    cleaned_inflammation = morphology.remove_small_objects(inflammation_regions, min_size=50)
    return cleaned_inflammation

def describe_features(predicted_mask):
    description = []
    
    if np.any(predicted_mask > 0.5):
        description.append("Tumor region detected")
        cleaned_inflammation = detect_inflammation(predicted_mask)
        if np.any(cleaned_inflammation):
            description.append("Inflammation detected")
    else:
        description.append("No tumor detected")
        description.append("No inflammation found")
        description.append("Healthy tissue observed")
    
    return description

def determine_brain_lobe(mask: np.ndarray) -> str:
    if mask.ndim > 2:
        mask = mask.squeeze()
    if mask.ndim > 2:
        mask = mask[:, :, 0]
        
    detector = BrainLobeDetector()
    binary_mask = (mask > 0.5).astype(np.uint8)
    primary_lobe, confidence, affected_regions = detector.detect_lobe(binary_mask)
    
    if primary_lobe == "No tumor detected":
        return "No tumor found"
    
    if confidence < 0.3:
        return f"Tumor detected in border region between {', '.join(affected_regions)}"
    
    return f"Tumor primarily in {primary_lobe} lobe ({confidence:.1%} confidence)"

def generate_description(feature_description, predicted_mask):
    if not np.any(predicted_mask > 0.5):
        return (
            "The MRI scan appears to show healthy brain tissue with no evidence of tumors "
            "or significant abnormalities. Continued routine monitoring is recommended.",
            0.0,
            0.0,
            "No tumor found"
        )
    
    tumor_size = np.sum(predicted_mask > 0.5) / (predicted_mask.shape[0] * predicted_mask.shape[1])
    inflammation_intensity = np.mean(predicted_mask[predicted_mask > 0.5]) if np.any(predicted_mask > 0.5) else 0
    brain_lobe = determine_brain_lobe(predicted_mask)
    
    description = DescriptionGenerator.generate_description(
        feature_description,
        tumor_size,
        inflammation_intensity,
        brain_lobe
    )
    
    return description, tumor_size, inflammation_intensity, brain_lobe

def visualize_images_with_features(x, predictions, feature_descriptions, descriptions):
    num_images = len(x)
    
    plt.figure(figsize=(12, num_images * 10))
    
    for j in range(num_images):
        img = x[j] / 255.0
        mask = predictions[j].squeeze()
        
        overlay = label2rgb(mask, image=img, bg_label=0, colors=[(0, 1, 0)], alpha=0.3)
        
        ax = plt.subplot(num_images, 1, j + 1)
        
        plt.imshow(overlay)
        plt.axis("off")
        
        title = "\n".join(feature_descriptions[j])
        ax.set_title(title, fontsize=16, color='red' if "Tumor region detected" in title else 'green')
        
        plt.text(0, -0.1, descriptions[j], fontsize=12, wrap=True, 
                transform=ax.transAxes, verticalalignment='top')
    
    plt.tight_layout()
    plt.show()

def analyze_mri_scan(image_path, model_path='unet_brain_mri_seg.keras'):
    """
    Analyze a single MRI scan and return the results.
    """
    # Load model
    model = keras.models.load_model(model_path, 
                                  custom_objects={'focal_tversky': focal_tversky, 'tversky': tversky})
    
    # Process single image
    x = np.asarray(load_img(image_path, target_size=(256, 256)))
    x = np.expand_dims(x, axis=0)
    disp_x = x.copy()
    
    # Make prediction
    pred = model.predict(x / 255)
    pred_t = (pred > 0.5).astype(np.uint8)
    
    # Generate analysis
    feature_description = describe_features(pred_t[0])
    description, tumor_size, inflammation_intensity, brain_lobe = generate_description(
        feature_description, pred_t[0]
    )
    
    # Visualize
    visualize_images_with_features(disp_x, pred_t, [feature_description], [description])
    
    return {
        'description': description,
        'features': feature_description,
        'tumor_size': tumor_size,
        'inflammation_intensity': inflammation_intensity,
        'brain_lobe': brain_lobe
    }

if __name__ == "__main__":
    # Example usage
    image_path = '/Users/sandeepkumar/Downloads/archive/lgg-mri-segmentation/kaggle_3m/TCGA_HT_8114_19981030/TCGA_HT_8114_19981030_19.tif'
    results = analyze_mri_scan(image_path)
    
    print("\nMRI Analysis Results:")
    print("-" * 50)
    print("\nDetailed Description:")
    print(results['description'])
    print("\nDetected Features:")
    for feature in results['features']:
        print(f"- {feature}")
    print("\nQuantitative Measurements:")
    print(f"- Tumor size: {results['tumor_size']:.1%} of visible brain area")
    print(f"- Inflammation intensity: {results['inflammation_intensity']:.2f} (0-1 scale)")
    print(f"- Affected brain region: {results['brain_lobe']}")
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 270ms/step
No description has been provided for this image
MRI Analysis Results:
--------------------------------------------------

Detailed Description:
The MRI scan reveals a very small tumor located in the Tumor primarily in right parietal lobe (33.0% confidence). There is severe inflammation surrounding the tumor region. Regular monitoring and follow-up imaging studies are recommended to track any changes in the tumor size or characteristics. The parietal lobe location may impact sensory processing and spatial awareness. 

Detected Features:
- Tumor region detected
- Inflammation detected

Quantitative Measurements:
- Tumor size: 3.1% of visible brain area
- Inflammation intensity: 1.00 (0-1 scale)
- Affected brain region: Tumor primarily in right parietal lobe (33.0% confidence)
In [ ]: